# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import re
import csv


requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告


class Fang_Get():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        # requests.utils.dict_from_cookiejar(html.cookies)  # 获取cookies转dict
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            # "User-Agent": ua.random
            # 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15'
            'authority': 'cd.esf.fang.com',
            'method': 'GET',
            'path': '/house/i32/',
            'scheme': 'https',
            'accept': ',',
            'accept-encoding': 'gzip, deflate, br',
            'accept-language': 'zh-CN,zh;q=0.9',
            'cookie': 'city=cd; global_cookie=3stc5s1v1l0lt839wrrqyybwk27k82owk3f; fang_hao123_layed=1; logGuid=9149efc6-de86-49fa-83ee-cb3c5bfba6be; g_sourcepage=esf_fy%5Elb_pc; Integrateactivity=notincludemc; unique_cookie=U_3stc5s1v1l0lt839wrrqyybwk27k82owk3f*2; __utma=147393320.844845003.1584860327.1584860327.1584860327.1; __utmc=147393320; __utmz=147393320.1584860327.1.1.utmcsr=cd.fang.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmt_t0=1; __utmt_t1=1; __utmt_t2=1; __utmb=147393320.3.10.1584860327; csrfToken=XUwwejRsxZ2f-gmmg1w7xYRQ',
            'dnt': '1',
            'referer': 'https://cd.esf.fang.com/',
            'sec-fetch-dest': 'document',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-site': 'same-origin',
            'sec-fetch-user': '?1',
            'upgrade-insecure-requests': '1',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'
        }


    def get_contents(self, cursor):
        url = "https://cd.esf.fang.com/house/i32/"
        html = HttpUtils.do_request("GET", url, self.headers, "")
        selector = etree.HTML(html.text)
        root = selector.xpath('//dl[@class="clearfix"]')
        for d in root:
            dict_data = dict()
            dict_data['url'] = d.xpath('.//h4/a/@href')[0]  # 链接
            dict_data['title'] = d.xpath('.//h4/a/span/text()')[0]  # 标题

            dict_data['tag'] = "|".join(d.xpath('.//p[@class="tel_shop"]/text()'))\
                .replace("\r", "").replace("\n", "").replace(" ", "").replace("�O", "㎡")  # 标签
            dict_data['area'] = d.xpath('.//p[@class="add_shop"]/a/@title')[0]  # 小区
            dict_data['address'] = d.xpath('.//p[@class="add_shop"]/span/text()')[0]  # 地址
            dict_data['ditie'] = "".join(d.xpath('.//span[@class="bg_none icon_dt"]/text()'))  # 地铁
            dict_data['price_1'] = d.xpath('.//dd[@class="price_right"]/span[1]/text()')[0]  # 总价
            dict_data['price_2'] = d.xpath('.//dd[@class="price_right"]/span[2]/text()')[0]\
                .replace("�O", "㎡")  # 单价
            print(str(dict_data))


if __name__ == '__main__':
    fang_get = Fang_Get()
    for i in range(1, 100):
        print(f"============{i}=============")
        fang_get.get_contents(i)